#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2024/2/21 11:51
# @Author  : 王凯
# @File    : dont_filter_request_parse_name.py
# @Project : scrapy_spider
from scrapy import signals


class DontFilterRequestParseNameSpiderMiddleware:

    def process_spider_output(self, response, result, spider):
        # Called with the results returned from the Spider, after
        # it has processed the response.
        dont_filter_request_parse_name_list = spider.settings.get("DONT_FILTER_REQUEST_PARSE_NAME_LIST", [])
        # Must return an iterable of Request, or item objects.
        for request in result:
            if hasattr(request, "callback"):
                callback_name = getattr(request.callback, "__name__", None)
                if callback_name:
                    if callback_name in dont_filter_request_parse_name_list:
                        request.dont_filter = True
            yield request

    def process_start_requests(self, start_requests, spider):
        # Called with the start requests of the spider, and works
        # similarly to the process_spider_output() method, except
        # that it doesn’t have a response associated.

        # Must return only requests (not items).
        for r in start_requests:
            r.dont_filter = True  # 首个请求不过滤
            yield r
